/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package application.externalServices;

/**
 *
 * @author aa6345 (based on example from http://java.sun.com/docs/books/tutorial/networking/urls/readingWriting.html)
 */
import java.net.*;
import java.io.*;
import java.net.URLEncoder;
import java.util.ArrayList;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;

public class YahooTermExtractor {

    private String fullURL;
    private String fullText;
    private String fullResponse;
    private ArrayList<String> extractedKeywords = new ArrayList();

    public void YahooTermExtractor() {
        this.fullText = "";
        this.fullURL = "";
        this.fullResponse = "";
        //this.extractedKeywords = null;
    }

    public void YahooTermExtractor(String thisText) {
        /*thisURL has to be URL Encoded*/
        //public static void main(String[] args) throws Exception {
        //URL yahoo = new URL("http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20search.termextract%20where%20context%3D%22Community%20participation%20in%20emergency%20planning%20in%20the%20UK%22&diagnostics=true&env=http%3A%2F%2Fgithub.com%2Fspullara%2Fyql-tables%2Fraw%2Fef685688d649a7514ebd27722366b2918d966573%2Falltables.env");
        this.setFullText(thisText);
        this.createURLWithText();
        this.extractedKeywords = null;
    }

    public void startTermExtract() {
        try {
            URL yahoo = new URL(this.getFullURL());
            URLConnection yc = yahoo.openConnection();
            BufferedReader in = new BufferedReader(
                    new InputStreamReader(
                    yc.getInputStream()));

            String inputLine;
            String fullTmpResponse = "";
            String[] tmpt;

            while ((inputLine = in.readLine()) != null) {
                //System.out.println(inputLine + "\n");
                if (!inputLine.contains("<!--")) {
                    tmpt = inputLine.split("<!--");
                    fullTmpResponse = fullTmpResponse + tmpt[0];
                } else {
                    fullTmpResponse = fullTmpResponse + inputLine;
                }
            }
            this.setFullXMLResponse(fullTmpResponse);
            in.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public void createURLWithText() {
        //String part1 = "http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20search.termextract%20where%20context%3D%22";
        //String part2 = "%22&diagnostics=true&env=http%3A%2F%2Fgithub.com%2Fspullara%2Fyql-tables%2Fraw%2Fef685688d649a7514ebd27722366b2918d966573%2Falltables.env";
        //String part1 = "http://search.yahooapis.com/ContentAnalysisService/V1/termExtraction?appid=SB95iv_V34E7I0.zExUWbzWfFdV1gDLb1seHYmr0TDCKcvsHG2dHFPawFOWk3dwwyr27ng--&context=";
        String part1 = "http://search.yahooapis.com/ContentAnalysisService/V1/termExtraction?appid=dj0yJmk9NXNhcE9mQXZTYWJyJmQ9WVdrOWJFNU5ha0psTkhFbWNHbzlNall6TURrNE1EWXkmcz1jb25zdW1lcnNlY3JldCZ4PWMz&context=";
        String part2 = "%22";
        //http://query.yahooapis.com/v1/public/yql?q=SELECT%20*%20FROM%20search.termextract%20WHERE%20context=%22Italian%20sculptors%20and%20painters%20of%20the%20renaissance%20favored%20the%20Virgin%20Mary%20for%20inspiration%22
        String encodedFullText = "";

        try {
            encodedFullText = URLEncoder.encode(this.getFullText().toLowerCase(), "UTF-8");
        } catch (Exception e) {
            e.printStackTrace();
        }

        //fix to trick yahoo if text is short.
        while(encodedFullText.length()<400) {
            encodedFullText = encodedFullText +"."+encodedFullText;
        }

        this.setFullURL(part1 + encodedFullText + part2);
        System.out.println(this.getFullURL());
    }

    public String getFullText() {
        return fullText;
    }

    public void setFullText(String fullText) {
        this.fullText = fullText;
    }

    public String getFullURL() {
        return fullURL;
    }

    public void setFullURL(String fullURL) {
        this.fullURL = fullURL;
    }

    public String getFullXMLResponse() {
        return fullResponse;
    }

    public void setFullXMLResponse(String fullResponse) {
        this.fullResponse = fullResponse;
    }

    public void addToExtractedKeywords(String s) {
        this.extractedKeywords.add(s);
    }

    public ArrayList getExtractedKeywords() {
        return this.extractedKeywords;
    }

    public void setExtractedKeywords(ArrayList extractedKeywords) {
        this.extractedKeywords = extractedKeywords;
    }

    public void extractXMLKeywords() {
        try {
            
            DocumentBuilderFactory factory =
                    DocumentBuilderFactory.newInstance();
            DocumentBuilder db = factory.newDocumentBuilder();
            InputSource inStream = new InputSource();
            inStream.setCharacterStream(new StringReader(this.getFullXMLResponse()));
            Document doc = db.parse(inStream);

            doc.getDocumentElement().normalize();

            NodeList nodeLst = doc.getElementsByTagName("ResultSet");
            //NodeList nodeLst = doc.getElementsByTagName("query"); //used with old termextractor url

            ArrayList<String> tmpArrayList = new ArrayList<String>();

            for (int s = 0; s < nodeLst.getLength(); s++) {
                Node fstNode = nodeLst.item(s);
                if (fstNode.getNodeType() == Node.ELEMENT_NODE) {
                    Element fstElmnt = (Element) fstNode;
                    //NodeList resultsElmntLst = fstElmnt.getElementsByTagName("results"); //used with old termextractor url
                    NodeList resultsElmntLst = fstElmnt.getElementsByTagName("Result");

                    for (int k = 0; k < resultsElmntLst.getLength(); k++) {

                        if (!tmpArrayList.contains(resultsElmntLst.item(k).getTextContent().trim())) {
                                    tmpArrayList.add(resultsElmntLst.item(k).getTextContent().trim());
                        }
                        /*
                        //used with old termextractor url
                        Node resultsNode = resultsElmntLst.item(k);
                        if (resultsNode.getNodeType() == Node.ELEMENT_NODE) {
                            Element resultsElmnt = (Element) resultsNode;
                            NodeList resultElmntLst = resultsElmnt.getElementsByTagName("Result");

                            for (int r = 0; r < resultElmntLst.getLength(); r++) {
                                if (!tmpArrayList.contains(resultElmntLst.item(r).getTextContent().trim())) {
                                    tmpArrayList.add(resultElmntLst.item(r).getTextContent().trim());
                                }
                            }
                        }
                         *
                         */
                    }

                }
            }

            /*
            for (int s = 0; s < nodeLst.getLength(); s++) {
            Node fstNode = nodeLst.item(s);
            if (fstNode.getNodeType() == Node.ELEMENT_NODE) {
            Element fstElmnt = (Element) fstNode;
            NodeList resultElmntLst = fstElmnt.getElementsByTagName("Result");

            for (int r = 0; r < resultElmntLst.getLength(); r++) {

            //if(!this.getExtractedKeywords().contains(resultElmntLst.item(r).getTextContent().trim())) {
            //    this.addToExtractedKeywords(resultElmntLst.item(r).getTextContent());
            //}
            if (!tmpArrayList.contains(resultElmntLst.item(r).getTextContent().trim())) {
            tmpArrayList.add(resultElmntLst.item(r).getTextContent().trim());
            }

            }

            }
            }
             *
             */

            //this.setExtractedKeywords(tmpArrayList);
            this.getExtractedKeywords().addAll(tmpArrayList);

        } catch (Exception e) {
            //System.out.println(e);
        }
        /*
        try {
        DocumentBuilderFactory factory =
        DocumentBuilderFactory.newInstance();
        DocumentBuilder db = factory.newDocumentBuilder();
        InputSource inStream = new InputSource();
        inStream.setCharacterStream(new StringReader(this.getFullXMLResponse()));
        Document doc = db.parse(inStream);

        doc.getDocumentElement().normalize();

        NodeList nodeLst = doc.getElementsByTagName("query");

        for (int s = 0; s < nodeLst.getLength(); s++) {
        Node fstNode = nodeLst.item(s);
        if (fstNode.getNodeType() == Node.ELEMENT_NODE) {
        Element fstElmnt = (Element) fstNode;
        NodeList resultsElmntLst = fstElmnt.getElementsByTagName("results");

        for (int r = 0; r < resultsElmntLst.getLength(); r++) {
        Node resultNode = resultsElmntLst.item(r);
        if (resultNode.getNodeType() == Node.ELEMENT_NODE) {
        Element resultElmnt = (Element) resultNode;
        NodeList resultElmntLst = resultElmnt.getElementsByTagName("Result");
        for (int p = 0; p < resultElmntLst.getLength(); p++) {
        //System.out.println("Node content: " +resultElmntLst.item(p).getTextContent()+" -> "+resultElmntLst.item(p).getNodeValue());
        if(!this.getExtractedKeywords().contains(resultElmntLst.item(p).getTextContent().trim())) {
        this.addToExtractedKeywords(resultElmntLst.item(p).getTextContent());
        }
        }

        }
        }
        }
        }
        } catch (Exception e) {
        //System.out.println(e);
        }
         *
         */
    }
}
